package com.smartdo.crawler.processor;

import cn.hutool.core.convert.Convert;
import cn.hutool.core.io.FileUtil;
import cn.hutool.json.JSONUtil;
import com.smartdo.crawler.utils.RedisFactory;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

/**
 * 1.输入需要爬取的卖家的URL列表
 * 2.采集每个卖家下有的上篇链接
 * 3.采集的商品链接放入reids资源池
 *
 * @author zhangcheng
 */
public class ListResourceCrawler implements PageProcessor {

    public static void main(String[] args) {
        List<String> urlList = FileUtil.readLines("E:\\data\\temp\\sumaitong\\第一批卖家.txt", "UTF-8");
        urlList.forEach(url -> {
            url = url.replace("//", "https://");
//        String url = "https://www.aliexpress.com/category/100003109/women-clothing.html?site=glo&brandValueIds=201639631&g=y&needQuery=n&tag=";
            String urlFormat = url.replace(".html?", "/%d.html?");
            int nowPageNum = 1;
            Request request1 = new Request(String.format(urlFormat, nowPageNum));
            request1.putExtra("urlFormat", urlFormat);
            request1.putExtra("baseUrl", url);
            request1.putExtra("nowPageNum", nowPageNum + 1);
            Spider.create(new ListResourceCrawler())
                    .addRequest(request1)
                    .run();
        });
    }


    @Override
    public void process(Page page) {
        List<String> urlList = page.getHtml().xpath("//*[@id=\"list-items\"]/ul/li/div/div[1]/div/a/@href").all();
        if (urlList.size() > 0) {
            urlList.forEach(url -> {
                url = url.replace("//", "https://");
                if (!RedisFactory.sismember("test:sumaitong:set", url)) {
                    RedisFactory.setListValue("test:sumaitong:list", url);
                    RedisFactory.setSetValue("test:sumaitong:set", url);
                }
            });
            page.addTargetRequest(builRequest(page.getRequest()));
        } else {
            RedisFactory.setHasValue("test:sumaitong:historyCrawl", page.getRequest().getExtra("baseUrl").toString(), JSONUtil.toJsonStr(page.getRequest().getExtras()));
        }
    }

    @Override
    public Site getSite() {
        return Site.me();
    }

    public Request builRequest(Request request) {
        String urlFormat = request.getExtra("urlFormat").toString();
        int nowPageNum = Convert.toInt(request.getExtra("nowPageNum"));
        Request request1 = new Request(String.format(urlFormat, nowPageNum));
        request1.putExtra("urlFormat", urlFormat);
        request1.putExtra("nowPageNum", nowPageNum + 1);
        request1.putExtra("baseUrl", request.getExtra("baseUrl"));
        return request1;
    }
}
